import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
from IPython.core.pylabtools import figsize

import seaborn as sns
import plotly.express as px

import numpy as np
import pandas as pd
import polars as pl

import statsmodels.formula.api as smf
import statsmodels.api as sm

import glob
import torch

from discriminationAnalysis import Full_Fisher_max_delta,\
Fisher_derivatives_faces_mean, Fisher_derivatives_faces_covariance, \
Fisher_derivatives_faces

from basicModel import EstimateAngle_Faces_final, EstimateAngle_Faces_experimental

from datageneration.faces.rotated_olivetti import FaceDataset

FI = {}

losses = ['linear', 'log', 'sqrt']
distributions= ['low_conc', 'high_conc', 'uniform']

for loss in losses:
    FI[loss] = {}
    for dist in distributions:
        for rep in range(3):
            ckpt = glob.glob(f'trainedParameters/Exp8/dropout/{loss}/{dist}/rep{rep}/epoch*')[0]
            model = EstimateAngle_Faces_final.load_from_checkpoint(ckpt, loss_pair=(loss, lambda d:d))
            
            baseline = np.linspace(0, 2*np.pi, 60)
            mean_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_mean)
            mean_outs = pd.DataFrame({'theta':mean_outs[0], 'Fisher':mean_outs[1]})
    
            cov_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_covariance)
            cov_outs = pd.DataFrame({'theta':cov_outs[0], 'Fisher':cov_outs[1]})
        
            FI[loss][f'{dist}{rep}mean'] = mean_outs
            FI[loss][f'{dist}{rep}cov'] = cov_outs
    
            mean_outs.to_csv(f'FI_saved/{loss}/{dist}{rep}mean', index=False)
            cov_outs.to_csv(f'FI_saved/{loss}/{dist}{rep}cov', index=False)

for rep in range(3):
    data = FI['linear'][f'low_conc{rep}mean']
    plt.plot(data['theta'], data['Fisher'])

for rep in range(3):
    data = FI['sqrt'][f'uniform{rep}mean']
    plt.plot(data['theta'], data['Fisher']**0.5)

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['linear'][f'{dist}{rep}mean']
        plt.plot(data['theta'], data['Fisher']**0.5, color=f'C{i}')

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['log'][f'{dist}{rep}mean']
        plt.plot(data['theta'], data['Fisher']**0.5, color=f'C{i}')

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['sqrt'][f'{dist}{rep}mean']
        plt.plot(data['theta'], data['Fisher']**0.5, color=f'C{i}')

for i, loss in enumerate(losses):
    for rep in range(3):
        data = FI[loss][f'uniform{rep}mean']
        plt.plot(data['theta'], data['Fisher']**0.5, color=f'C{i}')

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['linear'][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['log'][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['sqrt'][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

for i, loss in enumerate(losses):
    for rep in range(3):
        data = FI[loss][f'uniform{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['sqrt'][f'{dist}{rep}cov']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

total_Fisher = []
for loss in losses:
    for dist in distributions:
        for rep in range(3):
            data = FI[loss][f'{dist}{rep}mean']
            FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

            total_Fisher.append({'loss': loss, 'dist': dist, 'FI_total': FI_total})

total_Fisher = pd.DataFrame(total_Fisher)

sns.catplot(total_Fisher, x='dist', y='FI_total', hue='loss')
plt.title('Total Fisher information')

Text(0.5, 1.0, 'Total Fisher information')

variances = []
for loss in losses:
    for dist in distributions:
        for rep in range(3):
            data = FI[loss][f'{dist}{rep}mean']
            var = (data['Fisher']**0.5).var()
            total_diff = (data['Fisher']**0.5).max() - (data['Fisher']**0.5).min()

            variances.append({'loss': loss, 'dist': dist, 'variance': var, 'total_diff': total_diff})

variances = pd.DataFrame(variances)

sns.catplot(variances, x='dist', y='variance', hue='loss')

<seaborn.axisgrid.FacetGrid at 0x15e614e20>

sns.catplot(variances, x='dist', y='total_diff', hue='loss')

<seaborn.axisgrid.FacetGrid at 0x15e6184f0>

from scipy.signal import lombscargle

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['log'][f'{dist}{rep}mean']
        plt.plot(range(1,10), 
                 lombscargle(data['theta'], data['Fisher']**0.5-(data['Fisher']**0.5).mean(), range(1,10)),
                 color=f'C{i}' )

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['linear'][f'{dist}{rep}mean']
        plt.plot(range(1,10), 
                 lombscargle(data['theta'], data['Fisher']**0.5-(data['Fisher']**0.5).mean(), range(1,10)),
                 color=f'C{i}' )

for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI['sqrt'][f'{dist}{rep}mean']
        plt.plot(range(1,10), 
                 lombscargle(data['theta'], data['Fisher']**0.5-(data['Fisher']**0.5).mean(), range(1,10)),
                 color=f'C{i}' )

for i, dist in enumerate(['low_conc', 'uniform']):
    for rep in [0,2]:
        data = FI['linear'][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

for i, dist in enumerate(['low_conc', 'uniform']):
    for rep in [0,2]:
        data = FI['linear'][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(range(1,10), 
                 lombscargle(data['theta'], data['Fisher']**0.5-(data['Fisher']**0.5).mean(), range(1,10)),
                 color=f'C{i}' )

for i, dist in enumerate(['low_conc', 'uniform']):
    for rep in [0,1,2]:
        data = FI['linear'][f'{dist}{rep}cov']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

for i, dist in enumerate(['low_conc', 'uniform']):
    for rep in [0,1,2]:
        data = FI['linear'][f'{dist}{rep}cov']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(range(1,10), 
                 lombscargle(data['theta'], data['Fisher']**0.5-(data['Fisher']**0.5).mean(), range(1,10)),
                 color=f'C{i}' )

from scipy.stats import vonmises
vm0 = vonmises(1E-8, np.pi/2)
vm1 = vonmises(1, np.pi/2)
vm2 = vonmises(3, np.pi/2)
thetas= np.linspace(0, 2*np.pi, 100)

plt.plot(thetas, vm0.pdf(thetas))
plt.plot(thetas, vm1.pdf(thetas))
plt.plot(thetas, vm2.pdf(thetas))

[<matplotlib.lines.Line2D at 0x344776190>]

%run basicModel.py
from trainers import trainModel

uniformConfig = {
             'loc_tr': np.pi/2,
             'kappa_tr': 1E-8,
             'loc_val': np.pi/2,
             'kappa_val': 1E-8,
             'loc_test': np.pi/2,
              'kappa_test': 1E-8
             }

model = EstimateAngle_Faces_experimental( ('linear', lambda d: 2. - torch.mean(d)), **uniformConfig)

trainModel(model, 'trainedParameters/pilot/CELU')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 123 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
123 K     Trainable params
0         Non-trainable params
123 K     Total params
0.493     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

ckpt = 'trainedParameters/pilot/CELU/epoch=50-step=13056.ckpt'
model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt, loss_pair=('linear', '_'))

model.eval()
thetas = torch.linspace(-np.pi, np.pi, 101)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

results = model.forward(stimuli.to(model.device)).cpu().detach()

plt.plot(thetas, model.decodeAngles(results), '.');

baseline = np.linspace(0, 2*np.pi, 100)
mean_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_mean)

FI_total = np.trapz(mean_outs[1]**0.5, mean_outs[0])

plt.plot(mean_outs[0], mean_outs[1]**0.5 / FI_total, '.-')

[<matplotlib.lines.Line2D at 0x34977ec70>]

%run basicModel.py
from trainers import trainModel

<Figure size 640x480 with 0 Axes>

uniformConfig = {
             'loc_tr': np.pi/2,
             'kappa_tr': 1E-8,
             'loc_val': np.pi/2,
             'kappa_val': 1E-8,
             'loc_test': np.pi/2,
              'kappa_test': 1E-8
             }

model = EstimateAngle_Faces_experimental( ('linear', lambda d: 2. - torch.mean(d)), **uniformConfig)

trainModel(model, 'trainedParameters/pilot/CELU_big')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model = EstimateAngle_Faces_experimental.load_from_checkpoint('trainedParameters/pilot/CELU_big/epoch=55-step=14336.ckpt', loss_pair=('', ''))
model.eval()
thetas = torch.linspace(-np.pi, np.pi, 200)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

results = model.forward(stimuli.to(model.device)).cpu().detach()
plt.plot(thetas, model.decodeAngles(results), '.');

%run basicModel.py
from trainers import trainModel

<Figure size 640x480 with 0 Axes>

uniformConfig = {
             'loc_tr': np.pi/2,
             'kappa_tr': 1E-8,
             'loc_val': np.pi/2,
             'kappa_val': 1E-8,
             'loc_test': np.pi/2,
              'kappa_test': 1E-8
             }

model = EstimateAngle_Faces_experimental( ('linear', lambda d: 2. - torch.mean(d)), **uniformConfig)

trainModel(model, 'trainedParameters/pilot/CELU_big_gradclip')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model = EstimateAngle_Faces_experimental.load_from_checkpoint('trainedParameters/pilot/CELU_big_gradclip/epoch=6-step=1792.ckpt', loss_pair=('', ''))
model.eval()
thetas = torch.linspace(-np.pi, np.pi, 200)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

results = model.forward(stimuli.to(model.device)).cpu().detach()
plt.plot(thetas, model.decodeAngles(results), '.');

model = EstimateAngle_Faces_experimental.load_from_checkpoint('trainedParameters/pilot/CELU_big_gradclip/epoch=6-step=1792.ckpt', loss_pair=('', ''))
model.eval()
baseline = np.linspace(0, 2*np.pi, 100)
mean_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_mean)

FI_total = np.trapz(mean_outs[1]**0.5, mean_outs[0])
plt.plot(mean_outs[0], mean_outs[1]**0.5 / FI_total, '.-')

[<matplotlib.lines.Line2D at 0x323c9a970>]

thetas = torch.linspace(0, 2*np.pi, 101).repeat(160,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 160)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

cos_sim = torch.nn.CosineSimilarity()

def find_error(model):
    outputs = model.forward(stimuli.to(model.device)).cpu()
    encoded_inputs = model.encodeAngles(thetas)

    sim = cos_sim(outputs, encoded_inputs).reshape(101, 160).mean(dim=1)
    error = torch.arccos(torch.minimum(sim, torch.ones(sim.shape)))
    return error.detach().numpy()

loss = 'linear'

figsize(9,9)
plt.subplots(3,1)
plt.suptitle(f'Error - Loss: {loss}')

dists = ['high_conc', 'low_conc', 'uniform']

for i, dist in enumerate(dists):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )

        plt.subplot(3,1,i+1)
        plt.plot(theta_plot, find_error(model))

        plt.title(f'Distribution: {dist}')
        plt.ylim([0,2.])

loss = 'log'

figsize(9,9)
plt.subplots(3,1)
plt.suptitle(f'Error - Loss: {loss}')

dists = ['high_conc', 'low_conc', 'uniform']

for i, dist in enumerate(dists):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )

        plt.subplot(3,1,i+1)
        plt.plot(theta_plot, find_error(model))

        plt.title(f'Distribution: {dist}')
        plt.ylim([0,2.])

loss = 'sqrt'

figsize(9,9)
plt.subplots(3,1)
plt.suptitle(f'Error - Loss: {loss}')

dists = ['high_conc', 'low_conc', 'uniform']

for i, dist in enumerate(dists):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )

        plt.subplot(3,1,i+1)
        plt.plot(theta_plot, find_error(model))

        plt.title(f'Distribution: {dist}')
        plt.ylim([0,2.])

loss = 'linear'

figsize(9,9)
plt.subplots(3,1)
plt.suptitle(f'Error - Loss: {loss}')

dists = ['high_conc', 'low_conc', 'uniform']

for i, dist in enumerate(dists):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        plt.subplot(3,1,i+1)
        plt.plot(theta_plot, find_error(model))

        plt.title(f'Distribution: {dist}')
        plt.ylim([0,0.8])

loss = 'log'

figsize(9,9)
plt.subplots(3,1)
plt.suptitle(f'Error - Loss: {loss}')

dists = ['high_conc', 'low_conc', 'uniform']

for i, dist in enumerate(dists):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        plt.subplot(3,1,i+1)
        plt.plot(theta_plot, find_error(model))

        plt.title(f'Distribution: {dist}')
        plt.ylim([0,0.8])

loss = 'sqrt'

figsize(9,9)
plt.subplots(3,1)
plt.suptitle(f'Error - Loss: {loss}')

dists = ['high_conc', 'low_conc', 'uniform']

for i, dist in enumerate(dists):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        plt.subplot(3,1,i+1)
        plt.plot(theta_plot, find_error(model))

        plt.title(f'Distribution: {dist}')
        plt.ylim([0,0.8])

FI = {}

losses = ['linear', 'log', 'sqrt']
distributions= ['low_conc', 'high_conc', 'uniform']

for loss in losses:
    FI[loss] = {}
    for dist in distributions:
        for rep in range(3):
            ckpt = glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
            model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt, loss_pair=(loss, lambda d:d))
            
            baseline = np.linspace(0, 2*np.pi, 100)
            mean_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_mean)
            mean_outs = pd.DataFrame({'theta':mean_outs[0], 'Fisher':mean_outs[1]})
    
            total_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces)
            total_outs = pd.DataFrame({'theta':total_outs[0], 'Fisher':total_outs[1]})
        
            FI[loss][f'{dist}{rep}mean'] = mean_outs
            FI[loss][f'{dist}{rep}total'] = total_outs
    
            mean_outs.to_csv(f'FI_saved/CELU/{loss}/{dist}{rep}mean', index=False)
            total_outs.to_csv(f'FI_saved/CELU/{loss}/{dist}{rep}total', index=False)

figsize(5,5)
loss = 'linear'
for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')


plt.title(f'Fisher information, {loss} loss')

Text(0.5, 1.0, 'Fisher information, linear loss')

loss = 'log'
for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

plt.title(f'Fisher information, {loss} loss')

Text(0.5, 1.0, 'Fisher information, log loss')

loss = 'sqrt'
for i, dist in enumerate(distributions):
    for rep in range(3):
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
        
        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

plt.title(f'Fisher information, {loss} loss')

Text(0.5, 1.0, 'Fisher information, sqrt loss')

dist = 'uniform'
for i, loss in enumerate(losses):
    for rep in range(3):
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

dist = 'low_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        if loss == 'log' and rep == 0:
            continue
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

dist = 'high_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

dist = 'uniform'
for i, loss in enumerate(losses):
    for rep in range(3):
        for j in [0,1]:
            if j == 0:
                data = FI[loss][f'{dist}{rep}mean']
            if j == 1:
                data = FI[loss][f'{dist}{rep}total']
            
            FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
    
            plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{j}')

dist = 'low_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        for j in [0,1]:
            if j == 0:
                data = FI[loss][f'{dist}{rep}mean']
            if j == 1:
                data = FI[loss][f'{dist}{rep}total']
            FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
    
            plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{j}')

dist = 'high_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        for j in [0,1]:
            if j == 0:
                data = FI[loss][f'{dist}{rep}mean']
            if j == 1:
                data = FI[loss][f'{dist}{rep}total']
            
            FI_total = np.trapz(data['Fisher']**0.5, data['theta'])
    
            plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{j}')

thetas = torch.linspace(0, 2*np.pi, 101).repeat(160,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 160)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

cos_sim = torch.nn.CosineSimilarity()

def find_magnitude(model):
    outputs = model.forward(stimuli.to(model.device)).cpu()
    magnitude = torch.norm(outputs, dim=1)
    magnitude = magnitude.reshape((101, 160)).mean(dim=1)

    return magnitude.detach().numpy()



dist = 'high_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )

        plt.plot(theta_plot, find_magnitude(model), color=f'C{i}', label=loss)

plt.legend()

<matplotlib.legend.Legend at 0x3577bef70>

thetas = torch.linspace(0, 2*np.pi, 101).repeat(160,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 160)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

cos_sim = torch.nn.CosineSimilarity()

def find_magnitude(model):
    outputs = model.forward(stimuli.to(model.device)).cpu()
    magnitude = torch.norm(outputs, dim=1)
    magnitude = magnitude.reshape((101, 160)).mean(dim=1)

    return magnitude.detach().numpy()



dist = 'low_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )

        plt.plot(theta_plot, find_magnitude(model), color=f'C{i}', label=loss)

plt.legend()

<matplotlib.legend.Legend at 0x35215b730>

thetas = torch.linspace(0, 2*np.pi, 101).repeat(160,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 160)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

cos_sim = torch.nn.CosineSimilarity()

def find_magnitude(model):
    outputs = model.forward(stimuli.to(model.device)).cpu()
    magnitude = torch.norm(outputs, dim=1)
    magnitude = magnitude.reshape((101, 160)).mean(dim=1)

    return magnitude.detach().numpy()



dist = 'uniform'
for i, loss in enumerate(losses):
    for rep in range(3):
        ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )

        plt.plot(theta_plot, find_magnitude(model), color=f'C{i}', label=loss)

plt.legend()

<matplotlib.legend.Legend at 0x356e00cd0>

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 5)
plt.subplots(1,3)

dist = 'high_conc'
loss = 'linear'
for rep in range(3):
    ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
    model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                  loss_pair=('linear', lambda d:d)
                                                                  )
    outputs = model.forward(stimuli.to(model.device)).cpu().detach()

    plt.subplot(1,3,rep+1)
    plt.scatter(outputs[:,0], outputs[:,1], c=thetas)

thetas = torch.linspace(0, 2*np.pi, 101).repeat(160,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 160)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 5)
plt.subplots(1,3)

dist = 'high_conc'
loss = 'log'
for rep in range(3):
    ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
    model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                  loss_pair=('linear', lambda d:d)
                                                                  )
    outputs = model.forward(stimuli.to(model.device)).cpu().detach()

    plt.subplot(1,3,rep+1)
    plt.scatter(outputs[:,0], outputs[:,1], c=thetas)

thetas = torch.linspace(0, 2*np.pi, 101).repeat(160,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 160)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 5)
plt.subplots(1,3)

dist = 'high_conc'
loss = 'sqrt'
for rep in range(3):
    ckpt =  glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
    model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                  loss_pair=('linear', lambda d:d)
                                                                  )
    outputs = model.forward(stimuli.to(model.device)).cpu().detach()

    plt.subplot(1,3,rep+1)
    plt.scatter(outputs[:,0], outputs[:,1], c=thetas)

loss = 'linear'

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=thetas)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        plt.title(dist)

loss = 'log'

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=thetas)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        plt.title(dist)

loss = 'sqrt'

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

dist = 'high_conc'

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=thetas)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        plt.title(dist)

loss = 'linear'

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp8/dropout/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_final.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=thetas)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        plt.title(dist)

loss = 'log'

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp8/dropout/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_final.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=thetas)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        plt.title(dist)

loss = 'sqrt'

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)

stimuli = FaceDataset(thetas, split='test').images.contiguous()

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp8/dropout/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_final.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=thetas)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        plt.title(dist)

loss = 'linear'

thetas = torch.linspace(0, 2*np.pi, 101).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((101, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

thetas_d = np.pi/2 * torch.ones(80)
stim_d = FaceDataset(thetas_d, split='test').images.contiguous()
        

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
        out_d = model.forward(stim_d.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=thetas)
        plt.scatter(out_d[:,0], out_d[:,1], c='black')
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')

        encoding = model.encodeAngles(theta_plot)
        plt.scatter(encoding[:,0], encoding[:,1])
        
        plt.title(dist)

figsize(3,3)
plt.scatter(np.cos(theta_plot), np.sin(theta_plot), c=theta_plot)

<matplotlib.collections.PathCollection at 0x3be1e0940>

loss = 'linear'

thetas = torch.linspace(0, 2*np.pi, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

stimuli = stimuli[80*torch.arange(0, 100)]

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()

        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=theta_plot)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        
        plt.title(dist)

loss = 'linear'

thetas = torch.linspace(-np.pi, np.pi, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

stimuli = stimuli[80*torch.arange(0, 100)]

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                loss_pair=('linear', lambda d:d)
                                                                )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.plot(theta_plot, theta_plot, 'k')
        plt.plot(theta_plot, model.decodeAngles(outputs))
        
        plt.title(dist)

loss = 'linear'

thetas = torch.linspace(0, 2*np.pi, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

stimuli = stimuli[80*torch.arange(0, 100)]

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp8/dropout/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_final.load_from_checkpoint(ckpt,
                                                                      loss_pair=('linear', lambda d:d)
                                                                      )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()

        plt.subplot(3,3,3*i+rep+1)
        plt.scatter(outputs[:,0], outputs[:,1], c=theta_plot)
        plt.plot(np.cos(theta_plot), np.sin(theta_plot), 'k')
        
        plt.title(dist)

loss = 'linear'

thetas = torch.linspace(-np.pi, np.pi, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

stimuli = stimuli[80*torch.arange(0, 100)]

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp8/dropout/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_final.load_from_checkpoint(ckpt,
                                                                loss_pair=('linear', lambda d:d)
                                                                )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.plot(theta_plot, theta_plot, 'k')
        plt.plot(theta_plot, model.decodeAngles(outputs))
        
        plt.title(dist)

loss = 'linear'

thetas = torch.linspace(-np.pi, np.pi, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

stimuli = stimuli[ torch.concat([80*torch.arange(0, 100),
                                 80*torch.arange(0, 100) + 1,
                                 80*torch.arange(0, 100) +2
                                ])]

figsize(15, 15)
plt.subplots(3,3)
plt.suptitle(loss)

for i, dist in enumerate(['high_conc', 'low_conc', 'uniform']):
    for rep in range(3):
        ckpt =  glob.glob(
                    f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*'
                )[0]
        model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt,
                                                                loss_pair=('linear', lambda d:d)
                                                                )
        model.eval()
        outputs = model.forward(stimuli.to(model.device)).cpu().detach()
    
        plt.subplot(3,3,3*i+rep+1)
        plt.plot(theta_plot, theta_plot, 'k')
        plt.plot(theta_plot, model.decodeAngles(outputs[0:100]))
        plt.plot(theta_plot, model.decodeAngles(outputs[100:200]))
        plt.plot(theta_plot, model.decodeAngles(outputs[200:]))
        
        plt.title(dist)

%run basicModel.py

model = FaceAngle_Flat.load_from_checkpoint('trainedParameters/pilot/lineEmbedding/epoch=11-step=3072.ckpt',
                                            loss_pair=('linear', '_'))

model.setup()

plt.hist(model.trainingData[:]['angle'])
plt.title('Training Angles')

Text(0.5, 1.0, 'Training Angles')

model.hparams['kappa_tr']

1.0

test_data = model.testData[:]

outs = model.forward(test_data['image'].to(model.device)).cpu().detach()
embeddings = model.encodeAngles(test_data['angle'])

plt.plot(outs[:,0], outs[:,1], '.')
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x375b882e0>]

plt.plot(test_data['angle'], model.decodeAngles(outs)[0], '.')
plt.plot(np.linspace(-np.pi/2, np.pi/2), np.linspace(-np.pi/2, np.pi/2))

[<matplotlib.lines.Line2D at 0x37e506cd0>]

%run basicModel.py 
from trainers import trainModel
from experiments.parameters import uniformConfig, lossFns_decreasing

lossFns_decreasing

{'linear': <function experiments.parameters.<lambda>(d)>,
 'square': <function experiments.parameters.<lambda>(d)>,
 'sqrt': <function experiments.parameters.<lambda>(d)>}

model = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),  **uniformConfig)

trainModel(model, 'trainedParameters/pilot/line2/')

wandb: Currently logged in as: lrast. Use `wandb login --relogin` to force relogin

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

/Users/luke/Documents/angleFinetuning/basicModel.py:439: UserWarning: The use of `x.T` on tensors of dimension other than 2 to reverse their shape is deprecated and it will throw an error in a future release. Consider `x.mT` to transpose batches of matrices or `x.permute(*torch.arange(x.ndim - 1, -1, -1))` to reverse the dimensions of a tensor. (Triggered internally at /Users/runner/work/pytorch/pytorch/pytorch/aten/src/ATen/native/TensorShape.cpp:3641.)
  self.positive_diff = lambda x, y: torch.abs(project(x)[:,0] - project(y)[:,0]).T

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model = FaceAngle_Flat.load_from_checkpoint('trainedParameters/pilot/line2/epoch=155-step=39936.ckpt',
                                            loss_pair=('linear', lossFns_decreasing['linear']))

model.setup()

plt.hist(model.trainingData[:]['angle'])
plt.title('Training Angles')

Text(0.5, 1.0, 'Training Angles')

outs = model.forward(test_data['image'].to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=test_data['angle'])

embeddings = model.encodeAngles(test_data['angle'])
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x3233279a0>]

plt.plot(test_data['angle'], model.decodeAngles(outs)[0], '.')

[<matplotlib.lines.Line2D at 0x320f4d730>]

%run basicModel.py 
from trainers import trainModel
from experiments.parameters import uniformConfig, lossFns_decreasing

from experiments.parameters import concentratedConfig_1
model2 = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),  **concentratedConfig_1)

trainModel(model2, 'trainedParameters/pilot/line2/conc')

wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
wandb: Currently logged in as: lrast. Use `wandb login --relogin` to force relogin

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model2 = FaceAngle_Flat.load_from_checkpoint(
            'trainedParameters/pilot/line2/conc/epoch=742-step=190208.ckpt',
            loss_pair=('linear', lossFns_decreasing['linear'])
)
model2.setup()

plt.hist(model2.trainingData[:]['angle'])
plt.title('Training Angles')

Text(0.5, 1.0, 'Training Angles')

test = FaceDataset(np.linspace(-np.pi/2, np.pi/2, 400))
images = test[:]['image'].to(model2.device)
angles = test[:]['angle']

outs = model2.forward(images).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=angles)

embeddings = model2.encodeAngles(angles)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x342121250>]

plt.plot(angles, model2.decodeAngles(outs)[0], '.')

[<matplotlib.lines.Line2D at 0x341aa0af0>]

%run basicModel.py 
from trainers import trainModel
from experiments.parameters import lossFns_decreasing

from experiments.parameters import concentratedConfig_2
model3 = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),  **concentratedConfig_2)
trainModel(model3, 'trainedParameters/pilot/line2/lowconc')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model3 = FaceAngle_Flat.load_from_checkpoint(
            'trainedParameters/pilot/line2/lowconc/epoch=956-step=244992.ckpt',
            loss_pair=('linear', lossFns_decreasing['linear'])
)
model3.setup()

plt.hist(model3.trainingData[:]['angle'])
plt.title('Training Angles')

Text(0.5, 1.0, 'Training Angles')

outs = model3.forward(images).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=angles)

embeddings = model3.encodeAngles(angles)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x3860cd3a0>]

plt.plot(angles, model3.decodeAngles(outs)[0], '.')

[<matplotlib.lines.Line2D at 0x340c70730>]

thetas = torch.linspace(-np.pi/2, np.pi/2, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

outs = model3.decodeAngles( model3.forward(stimuli.to(model3.device)) ).detach().cpu()

sqerr = (outs - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x38843ca90>]

out_high_conc = model2.decodeAngles( model2.forward(stimuli.to(model2.device)) ).detach().cpu()
sqerr = (out_high_conc - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x388661f70>]

for m in [model, model2, model3]:
    out = m.decodeAngles( m.forward(stimuli.to(m.device)) ).detach().cpu()
    sqerr = (out - thetas)**2
    MSE = sqerr.reshape((100, 80)).mean(dim=1)
    
    plt.plot(theta_plot, MSE)

smooth_model = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),
                             hidden_dims=(30, 20),
                             **concentratedConfig_2
                            )

trainModel(smooth_model, 'trainedParameters/pilot/line2/small_smooth')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

smooth_model = FaceAngle_Flat.load_from_checkpoint(
            'trainedParameters/pilot/line2/lowconc1/epoch=443-step=113664.ckpt',
            loss_pair=('linear', lossFns_decreasing['linear']),
            hidden_dims=(60, 20)
)

smooth_model

FaceAngle_Flat(
  (model): Sequential(
    (0): Linear(in_features=4096, out_features=60, bias=True)
    (1): LeakyReLU(negative_slope=0.01)
    (2): Linear(in_features=60, out_features=20, bias=True)
    (3): LeakyReLU(negative_slope=0.01)
    (4): Linear(in_features=20, out_features=2, bias=True)
  )
  (cosSim): CosineSimilarity()
)

out = smooth_model.decodeAngles( smooth_model.forward(stimuli.to(smooth_model.device)) ).detach().cpu()
sqerr = (out - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x389562100>]

plt.plot(theta_plot, out.reshape((100, 80)).var(1))

[<matplotlib.lines.Line2D at 0x3890fe9a0>]

plt.plot(theta_plot, out.reshape((100, 80)).mean(1) - theta_plot)

[<matplotlib.lines.Line2D at 0x3890fbee0>]

smooth_model.model[0](torch.randn(1,4096).to(smooth_model.device)).shape

torch.Size([1, 60])

%run basicModel.py
from trainers import trainModel
from experiments.parameters import lossFns_decreasing, concentratedConfig_2

smooth_model = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),
                             hidden_dims=(30, 20),
                             **concentratedConfig_2
                            )

smooth_model

FaceAngle_Flat(
  (model): Sequential(
    (0): Linear(in_features=4096, out_features=30, bias=True)
    (1): LeakyReLU(negative_slope=0.01)
    (2): Linear(in_features=30, out_features=20, bias=True)
    (3): LeakyReLU(negative_slope=0.01)
    (4): Linear(in_features=20, out_features=2, bias=True)
  )
  (cosSim): CosineSimilarity()
)

trainModel(smooth_model, 'trainedParameters/pilot/line2/small_smooth')

wandb: Using wandb-core as the SDK backend. Please refer to https://wandb.me/wandb-core for more information.
wandb: Currently logged in as: lrast. Use `wandb login --relogin` to force relogin

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/callbacks/model_checkpoint.py:653: Checkpoint directory /Users/luke/Documents/angleFinetuning/trainedParameters/pilot/line2/small_smooth exists and is not empty.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 123 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
123 K     Trainable params
0         Non-trainable params
123 K     Total params
0.494     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

smooth_model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/small_smooth/epoch=324-step=83200.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear']))

thetas = torch.linspace(-np.pi/2, np.pi/2, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

out = smooth_model.decodeAngles( smooth_model.forward(stimuli.to(smooth_model.device)) ).detach().cpu()
sqerr = (out - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x32242d6d0>]

smooth_model.setup()

plt.hist(smooth_model.trainingData[:]['angle'])

(array([ 522.,  291.,  244.,  293.,  437.,  884., 1458., 1736., 1456.,
         871.]),
 array([-1.57045877e+00, -1.25635231e+00, -9.42245901e-01, -6.28139436e-01,
        -3.14033002e-01,  7.34329224e-05,  3.14179868e-01,  6.28286302e-01,
         9.42392766e-01,  1.25649917e+00,  1.57060564e+00]),
 <BarContainer object of 10 artists>)

%run basicModel.py
from trainers import trainModel
from experiments.parameters import lossFns_decreasing, concentratedConfig_2

<Figure size 640x480 with 0 Axes>

less_data_model = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),
                             train_val_test_split=(200, 100, 100),
                             **concentratedConfig_2
                            )

trainModel(less_data_model, 'trainedParameters/pilot/line2/less_data')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

out = less_data_model.decodeAngles( less_data_model.forward(stimuli.to(less_data_model.device)) ).detach().cpu()
sqerr = (out - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x3141ab070>]

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200)
uniform_stim = FaceDataset(uniform_thetas, split='test').images.contiguous()


outs = less_data_model.forward(uniform_stim.to(less_data_model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x34226c100>]

def project(embeddings):
    unit_vector = torch.ones(2,1, device=embeddings.device) / 2**0.5
    return (embeddings @ unit_vector) @ unit_vector.T

plt.plot(uniform_thetas, project(outs).detach())

[<matplotlib.lines.Line2D at 0x341eb7910>,
 <matplotlib.lines.Line2D at 0x341eb7b20>]

%run basicModel.py
from trainers import trainModel
from experiments.parameters import lossFns_decreasing, concentratedConfig_2

limited_outs = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),
                             train_val_test_split=(200, 100, 100),
                             **concentratedConfig_2
                            )

trainModel(limited_outs, 'trainedParameters/pilot/line2/restricted')

VBox(children=(Label(value='Waiting for wandb.init()...\r'), FloatProgress(value=0.011168753711131608, max=1.0…

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/restricted/epoch=294-step=75520.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear'])
)

thetas = torch.linspace(-np.pi/2, np.pi/2, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

out = model.decodeAngles( model.forward(stimuli.to(model.device)) ).detach().cpu()
sqerr = (out - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x3906379d0>]

out.reshape((100, 80))[-1] - np.pi/2

tensor([-0.0499, -0.0801, -0.0645, -0.0407, -0.0172, -0.0118, -0.0501, -0.0374,
        -0.0374, -0.0339, -0.0143, -0.0516, -0.0233, -0.0332, -0.0247, -0.0434,
        -0.0903, -0.0459, -0.0689, -0.0422, -0.0313, -0.0210, -0.0417, -0.0601,
        -0.0220, -0.0633, -0.0643, -0.0314, -0.0573, -0.0302, -0.0416, -0.2265,
        -0.0310, -0.0701, -0.0330, -0.0360, -0.0161, -0.0317, -0.0257, -0.1158,
        -0.0467, -0.0808, -0.0418, -0.0423, -0.1218, -0.0451, -0.0528, -0.1071,
        -0.0316, -0.0403, -0.0595, -0.0433, -0.0409, -0.0692, -0.0184, -0.0956,
        -0.0499, -0.0608, -0.0828, -0.0273, -0.0301, -0.0141, -0.0156, -0.0741,
        -0.0670, -0.8382, -0.0142, -0.0255, -0.0119, -0.0209, -0.0161, -0.0482,
        -0.1189, -0.0518, -0.0186, -0.0486, -0.0298, -0.0219, -0.0443, -0.0457])

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200)
uniform_stim = FaceDataset(uniform_thetas, split='test').images.contiguous()


outs = model.forward(uniform_stim.to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x3202d2fd0>]

model.setup()

plt.hist(model.trainingData[:]['angle'])

(array([ 522.,  291.,  244.,  293.,  437.,  884., 1458., 1736., 1456.,
         871.]),
 array([-1.57045877e+00, -1.25635231e+00, -9.42245901e-01, -6.28139436e-01,
        -3.14033002e-01,  7.34329224e-05,  3.14179868e-01,  6.28286302e-01,
         9.42392766e-01,  1.25649917e+00,  1.57060564e+00]),
 <BarContainer object of 10 artists>)

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200).repeat(80,1)
uniform_thetas = uniform_thetas.T.reshape(-1)
uniform_stim = FaceDataset(uniform_thetas, split='test').images.contiguous()


outs = model.forward(uniform_stim.to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x3905c21c0>]

%run basicModel.py
from trainers import trainModel
from experiments.parameters import lossFns_decreasing, concentratedConfig_2

hard_limited_outs = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),
                             train_val_test_split=(200, 100, 100),
                             **concentratedConfig_2
                            )

trainModel(hard_limited_outs, 'trainedParameters/pilot/line2/restricted2')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/restricted2/epoch=502-step=128768.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear'])
)

thetas = torch.linspace(-np.pi/2, np.pi/2, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

out = model.decodeAngles( model.forward(stimuli.to(model.device)) ).detach().cpu()
sqerr = (out - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x3cc031f10>]

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200).repeat(80,1)
uniform_thetas = uniform_thetas.T.reshape(-1)
uniform_stim = FaceDataset(uniform_thetas, split='test').images.contiguous()


outs = model.forward(uniform_stim.to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x3cc0578e0>]

%run basicModel.py

<Figure size 640x480 with 0 Axes>

f = customTanh(np.pi/2)
fp = customTanh(np.pi/2 + 0.1, scaling=0.5)

xs = torch.linspace(-10, 10, 50)
plt.plot(xs, f(xs))
plt.plot(xs, fp(xs))

[<matplotlib.lines.Line2D at 0x3ccfce610>]

one_more = FaceAngle_Flat(('linear', lossFns_decreasing['linear']),
                             train_val_test_split=(200, 100, 100),
                             **concentratedConfig_2
                            )

trainModel(one_more, 'trainedParameters/pilot/line2/restricted3')

GPU available: True (mps), used: True
TPU available: False, using: 0 TPU cores
IPU available: False, using: 0 IPUs
HPU available: False, using: 0 HPUs
/Users/luke/Library/Python/3.9/lib/python/site-packages/pytorch_lightning/loggers/wandb.py:391: There is a wandb run already in progress and newly created instances of `WandbLogger` will reuse this run. If this is not desired, call `wandb.finish()` before instantiating `WandbLogger`.

  | Name   | Type             | Params
--------------------------------------------
0 | model  | Sequential       | 247 K 
1 | cosSim | CosineSimilarity | 0     
--------------------------------------------
247 K     Trainable params
0         Non-trainable params
247 K     Total params
0.988     Total estimated model params size (MB)

Sanity Checking: |                                        | 0/? [00:00<?, ?it/s]

Training: |                                               | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

Validation: |                                             | 0/? [00:00<?, ?it/s]

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/restricted3/epoch=508-step=130304.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear'])
)

thetas = torch.linspace(-np.pi/2, np.pi/2, 100).repeat(80,1)
thetas = thetas.T.reshape(-1)
theta_plot = thetas.reshape((100, 80)).mean(dim=1)
stimuli = FaceDataset(thetas, split='test').images.contiguous()

out = model.decodeAngles( model.forward(stimuli.to(model.device)) ).detach().cpu()
sqerr = (out - thetas)**2
MSE = sqerr.reshape((100, 80)).mean(dim=1)

plt.plot(theta_plot, MSE)

[<matplotlib.lines.Line2D at 0x3dcabbb50>]

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200).repeat(80,1)
uniform_thetas = uniform_thetas.T.reshape(-1)
uniform_stim = FaceDataset(uniform_thetas, split='test').images.contiguous()


outs = model.forward(uniform_stim.to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x390583a00>]

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/restricted3/epoch=508-step=130304.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear'])
)

baseline = np.linspace(-np.pi/2, np.pi/2, 100)
FI = Full_Fisher_max_delta(model, baseline, lambda m, t: Fisher_derivatives_faces_mean(m,t, num_samples=1))

total = np.trapz(FI[1]**0.5,FI[0])
plt.plot(FI[0], FI[1]**0.5/total)
plt.title('Extended tanh Fisher')

Text(0.5, 1.0, 'Extended tanh Fisher')

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/restricted2/epoch=502-step=128768.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear'])
)

baseline = np.linspace(-np.pi/2, np.pi/2, 100)
FI = Full_Fisher_max_delta(model, baseline, lambda m, t: Fisher_derivatives_faces_mean(m,t, num_samples=1))

total = np.trapz(FI[1]**0.5,FI[0])
plt.plot(FI[0], FI[1]**0.5/total)
plt.title('Hard tanh Fisher')

Text(0.5, 1.0, 'Hard tanh Fisher')

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/restricted/epoch=294-step=75520.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear'])
)

baseline = np.linspace(-np.pi/2, np.pi/2, 100)
FI = Full_Fisher_max_delta(model, baseline, lambda m, t: Fisher_derivatives_faces_mean(m,t, num_samples=1))

total = np.trapz(FI[1]**0.5,FI[0])
plt.plot(FI[0], FI[1]**0.5/total)
plt.title('Tanh Fisher')

Text(0.5, 1.0, 'Tanh Fisher')

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200).repeat(80,1)
uniform_thetas = uniform_thetas.T.reshape(-1)
uniform_stim = FaceDataset(uniform_thetas, split='test').images.contiguous()


outs = model.forward(uniform_stim.to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])

[<matplotlib.lines.Line2D at 0x33e27bee0>]

i = 16
single = outs.reshape(200, 80, 2)[:,i,:]
plt.scatter(single[:,0], single[:,1], c=uniform_thetas.reshape(200, 80)[:,i])

<matplotlib.collections.PathCollection at 0x3498202e0>

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/less_data/epoch=294-step=75520.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear']))

baseline = np.linspace(-np.pi/2, np.pi/2, 100)
FI = Full_Fisher_max_delta(model, baseline, lambda m, t: Fisher_derivatives_faces_mean(m,t, num_samples=1))

total = np.trapz(FI[1]**0.5,FI[0])
plt.plot(FI[0], FI[1]**0.5/total)
plt.title('No non-linearity Fisher')

Text(0.5, 1.0, 'No non-linearity Fisher')

model = FaceAngle_Flat.load_from_checkpoint(
            'trainedParameters/pilot/line2/lowconc/epoch=956-step=244992.ckpt',
            loss_pair=('linear', lossFns_decreasing['linear'])
)
baseline = np.linspace(-np.pi/2, np.pi/2, 100)
FI = Full_Fisher_max_delta(model, baseline, lambda m, t: Fisher_derivatives_faces_mean(m,t, num_samples=1))

total = np.trapz(FI[1]**0.5,FI[0])
plt.plot(FI[0], FI[1]**0.5/total)
plt.title('No non-linearity Fisher - large data')

Text(0.5, 1.0, 'No non-linearity Fisher - large data')

model = FaceAngle_Flat.load_from_checkpoint(
    'trainedParameters/pilot/line2/restricted/epoch=294-step=75520.ckpt',
    loss_pair = ('linear', lossFns_decreasing['linear'])
)

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200).repeat(100,1)
uniform_thetas = uniform_thetas.T.reshape(-1)
uniform_stim = FaceDataset(uniform_thetas, split='test', numbers=(200, 100, 100)).images.contiguous()

outs = model.forward(uniform_stim.to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])
plt.title('Test set embeddings')

Text(0.5, 1.0, 'Test set embeddings')

uniform_thetas = torch.linspace(-np.pi/2, np.pi/2, 200).repeat(200,1)
uniform_thetas = uniform_thetas.T.reshape(-1)
uniform_stim = FaceDataset(uniform_thetas, split='train', numbers=(200, 100, 100)).images.contiguous()


outs = model.forward(uniform_stim.to(model.device)).cpu().detach()
plt.scatter(outs[:,0], outs[:,1], c=uniform_thetas)

embeddings = less_data_model.encodeAngles(uniform_thetas)
plt.plot(embeddings[:,0], embeddings[:,1])
plt.title('Train set embeddings')

Text(0.5, 1.0, 'Train set embeddings')

def Fisher_derivatives_faces_mean_test(model, thetas, num_samples=1, image_delta=0.05):
    """ The mean term in the Fisher information, as evaluated by derivatives of the neural network
    """
    def point_Fisher(theta):
        """ This could definitely be sped up by vectorization in a variety of dimensions """
        I0 = FaceDataset(torch.zeros(100) + theta, split='test', numbers=(200, 100, 100)).images.contiguous()
        I0.requires_grad = True

        outputs = model.forward(I0.to(model.device).repeat((num_samples, 1, 1))).cpu()
        model_grad0 = torch.autograd.grad(outputs.mean(0)[0], I0, retain_graph=True)[0]
        model_grad1 = torch.autograd.grad(outputs.mean(0)[1], I0)[0]

        plus_I0 = FaceDataset(torch.zeros(100) + theta + image_delta/2, split='test',
                              numbers=(200, 100, 100)).images.contiguous()
        minus_I0 = FaceDataset(torch.zeros(100) + theta - image_delta/2, split='test',
                              numbers=(200, 100, 100)).images.contiguous()
        image_deriv = (plus_I0 - minus_I0) / image_delta 

        full_deriv0 = (model_grad0 * image_deriv).sum().item()
        full_deriv1 = (model_grad1 * image_deriv).sum().item()
        douts = torch.tensor([full_deriv0, full_deriv1])

        cov = outputs.T.cov().detach()
        torch.mps.empty_cache()
        return (douts @ torch.linalg.inv(cov) @ douts).item()

    FIs = np.zeros(len(thetas))
    for i, theta in enumerate(thetas):
        FIs[i] = point_Fisher(theta)

    return FIs

def Fisher_derivatives_faces_mean_train(model, thetas, num_samples=1, image_delta=0.05):
    """ The mean term in the Fisher information, as evaluated by derivatives of the neural network
    """
    def point_Fisher(theta):
        """ This could definitely be sped up by vectorization in a variety of dimensions """
        I0 = FaceDataset(torch.zeros(200) + theta, split='train', numbers=(200, 100, 100)).images.contiguous()
        I0.requires_grad = True

        outputs = model.forward(I0.to(model.device).repeat((num_samples, 1, 1))).cpu()
        model_grad0 = torch.autograd.grad(outputs.mean(0)[0], I0, retain_graph=True)[0]
        model_grad1 = torch.autograd.grad(outputs.mean(0)[1], I0)[0]

        plus_I0 = FaceDataset(torch.zeros(200) + theta + image_delta/2, split='train',
                              numbers=(200, 100, 100)).images.contiguous()
        minus_I0 = FaceDataset(torch.zeros(200) + theta - image_delta/2, split='train',
                              numbers=(200, 100, 100)).images.contiguous()
        image_deriv = (plus_I0 - minus_I0) / image_delta 

        full_deriv0 = (model_grad0 * image_deriv).sum().item()
        full_deriv1 = (model_grad1 * image_deriv).sum().item()
        douts = torch.tensor([full_deriv0, full_deriv1])

        cov = outputs.T.cov().detach()
        torch.mps.empty_cache()
        return (douts @ torch.linalg.inv(cov) @ douts).item()

    FIs = np.zeros(len(thetas))
    for i, theta in enumerate(thetas):
        FIs[i] = point_Fisher(theta)

    return FIs

baseline = np.linspace(-np.pi/2, np.pi/2, 100)
FI = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_mean_test)

total = np.trapz(FI[1]**0.5,FI[0])
plt.plot(FI[0], FI[1]**0.5/total)
plt.title('Fisher Information - test set')

Text(0.5, 1.0, 'Fisher Information - test set')

baseline = np.linspace(-np.pi/2, np.pi/2, 100)
FI = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_mean_train)

total = np.trapz(FI[1]**0.5,FI[0])
plt.plot(FI[0], FI[1]**0.5/total)
plt.title('Fisher Information - training set')

Text(0.5, 1.0, 'Fisher Information - training set')

def Fisher_derivatives_faces_mean_train(model, thetas, num_samples=1000, image_delta=0.05):
    """ The mean term in the Fisher information, as evaluated by derivatives of the neural network
    """
    def point_Fisher(theta):
        """ This could definitely be sped up by vectorization in a variety of dimensions """
        I0 = FaceDataset(torch.zeros(256) + theta, split='train').images.contiguous()
        I0.requires_grad = True

        outputs = model.forward(I0.to(model.device).repeat((num_samples, 1, 1))).cpu()
        model_grad0 = torch.autograd.grad(outputs.mean(0)[0], I0, retain_graph=True)[0]
        model_grad1 = torch.autograd.grad(outputs.mean(0)[1], I0)[0]

        plus_I0 = FaceDataset(torch.zeros(256) + theta + image_delta/2, split='train',
                              ).images.contiguous()
        minus_I0 = FaceDataset(torch.zeros(256) + theta - image_delta/2, split='train',
                               ).images.contiguous()
        image_deriv = (plus_I0 - minus_I0) / image_delta 

        full_deriv0 = (model_grad0 * image_deriv).sum().item()
        full_deriv1 = (model_grad1 * image_deriv).sum().item()
        douts = torch.tensor([full_deriv0, full_deriv1])

        cov = outputs.T.cov().detach()
        torch.mps.empty_cache()
        return (douts @ torch.linalg.inv(cov) @ douts).item()

    FIs = np.zeros(len(thetas))
    for i, theta in enumerate(thetas):
        FIs[i] = point_Fisher(theta)

    return FIs

FI = {}

losses = ['linear', 'log', 'sqrt']
distributions= ['low_conc', 'high_conc', 'uniform']

for loss in losses:
    FI[loss] = {}
    for dist in distributions:
        for rep in range(3):
            ckpt = glob.glob(f'trainedParameters/Exp10/bigCELU/{loss}/{dist}/rep{rep}/epoch*')[0]
            model = EstimateAngle_Faces_experimental.load_from_checkpoint(ckpt, loss_pair=(loss, lambda d:d))
            
            baseline = np.linspace(0, 2*np.pi, 100)
            mean_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces_mean_train)
            mean_outs = pd.DataFrame({'theta':mean_outs[0], 'Fisher':mean_outs[1]})
    
            total_outs = Full_Fisher_max_delta(model, baseline, Fisher_derivatives_faces)
            total_outs = pd.DataFrame({'theta':total_outs[0], 'Fisher':total_outs[1]})
        
            FI[loss][f'{dist}{rep}mean'] = mean_outs
            FI[loss][f'{dist}{rep}total'] = total_outs
    
            mean_outs.to_csv(f'FI_saved/CELU/{loss}/{dist}{rep}mean', index=False)
            total_outs.to_csv(f'FI_saved/CELU/{loss}/{dist}{rep}total', index=False)

dist = 'uniform'
for i, loss in enumerate(losses):
    for rep in range(3):
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

dist = 'low_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        if loss == 'log' and rep == 0:
            continue
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

dist = 'high_conc'
for i, loss in enumerate(losses):
    for rep in range(3):
        data = FI[loss][f'{dist}{rep}mean']
        FI_total = np.trapz(data['Fisher']**0.5, data['theta'])

        plt.plot(data['theta'], data['Fisher']**0.5 / FI_total, color=f'C{i}')

Linear Val Loss	█▄▃▂▂▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁▂▂▁▁▁▁▂
Train Loss	█▃█▃▄▄▃▄▁▄▂▂▆▂▁▂▄▂▁▃▂▅▄▃▁▁▃▅▂▃▂▃▂▂▂▂▄▄▃▂
Val Loss	█▄▃▂▂▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁▂▂▁▁▁▁▂
epoch	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
trainer/global_step	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███

Linear Val Loss	0.13648
Train Loss	0.35492
Val Loss	0.13648
epoch	250
trainer/global_step	64255

Linear Val Loss	▂▅██▄██▁▁▁▁▁▁▁▁▃▃▄▄▄▅▄▃▄▄▅▄▄▄▄▅▄▄▃▃▄▄▄▄▄
Train Loss	▇▆█▅▇▇▆▅▄▄▁▁▂▃▂▅▆▃▆▄▄▅▄▃▄▄▄▅▆▄▅▃▄▅▄▄▅▆▁▆
Val Loss	▂▅██▄██▁▁▁▁▁▁▁▁▃▃▄▄▄▅▄▃▄▄▅▄▄▄▄▅▄▄▃▃▄▄▄▄▄
epoch	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
trainer/global_step	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███

Linear Val Loss	0.47704
Train Loss	0.49272
Val Loss	0.47704
epoch	255
trainer/global_step	65535

Linear Val Loss	█▃▂▂▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▂▁▁▁
Train Loss	▆█▃▅▄▂▄▄▃▄▃▄▂▆▄▁▂▄▆▄▃▄▂▂▆▁▄▅▄▃▂▄▃▂▃▂▄▁▄▁
Val Loss	█▃▂▂▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▂▁▁▁
epoch	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
trainer/global_step	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███

Linear Val Loss	0.06785
Train Loss	0.20753
Val Loss	0.06785
epoch	206
trainer/global_step	52991

Linear Val Loss	▂█▁███████▁██▁███▁███████▁▁██████▁██▁███
Train Loss	█▃▄▂▂▃▂▂▂▂▂▂▂▁▂▂▂▂▂▁▁▂▁▁▂▂▁▁▂▂▁▂▁▁▁▁▂▂▁▁
Val Loss	█▂▃▃▂▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▁▁▁▁▁▁▁▁▁▂▁▂▁▁▁▁
epoch	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███
trainer/global_step	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇███

Linear Val Loss	1.0
Train Loss	0.01088
Val Loss	0.03148
epoch	355
trainer/global_step	91135

Linear Val Loss	▇▅▇▆▃█▂▃▃▃▄▂▄▃▂▂▂▂▄▂▁▁▁▁▁▄▁▂▂▁▃▁▁▄▂▂▂▂▂▁
Train Loss	▇▇█▅▃▄▅▆▃▄▂▃▄▄▃▂▂▃▂▃▂▇▄▂▂▂▂▂▂▂▂▂▁▂▁▁▂▁▁▂
Val Loss	█▆▆▃▃▃▂▃▃▃▂▃▂▂▁▁▁▂▁▃▁▂▁▁▂▂▁▁▂▂▁▁▂▃▁▁▂▁▁▂
epoch	▁▁▁▁▁▁▁▂▂▂▂▂▂▃▃▃▃▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇▇▇██
holder	█▁█▁████▁███████████████████████████████
trainer/global_step	▁▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▄▅▅▆▆▆▆▆▆▆▆▇▇▇▇▇▇▇█

Linear Val Loss	0.02636
Train Loss	0.01195
Val Loss	0.02636
epoch	942
holder	1
trainer/global_step	241407

Linear Val Loss	█▃▂▂▂▃▂▂▂▂▂▂▂▂▂▁▂▁▂▁▁▁▁▁▁▂▁▂▁▁▂▁▁▁▁▁▁▂▂▂
Train Loss	▆▆█▄▄▄▅▅▄▄▅▂▃▃▃▂▆▂▃▂▃▃▂▄▃▁▃▃▂▃▁▂▃▃▃▂▁▁▂▂
Val Loss	█▄▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▂▂▁▂▁▁▂▂▂▂▁▁▂▁▁▂▁▂▁▁▂▁▁
epoch	▁▁▁▁▁▂▂▃▃▃▃▃▃▃▃▄▄▄▄▄▄▄▄▅▅▅▅▆▆▆▆▇▇▇▇█████
holder	▁███████▁█████▁█████████████████████████
trainer/global_step	▁▁▁▁▁▂▂▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▄▄▅▅▅▅▅▅▅▆▆▇▇▇██

Linear Val Loss	0.02559
Train Loss	0.01218
Val Loss	0.02559
epoch	1156
holder	1
trainer/global_step	296191

Linear Val Loss	▆▄▅▅▂▄▃█▃▂▂▃▄▂▃▃▁▂▃▂▂▂▁▂▃▃▂▂▂▂▃▁▁▂▁▂▄▂▁▃
Train Loss	█▇▄▃▂▆▄▄▄▂▁▂▁▃▂▂▂▂▁▂▂▃▂▁▁▃▂▂▁▂▁▂▂▁▁▂▂▁▁▁
Val Loss	█▅▄▅▅▄▃▆▂▃▂▂▂▂▂▁▃▂▁▂▁▁▂▁▂▂▁▂▁▂▂▂▁▂▃▂▁▁▁▂
epoch	▁▂▂▂▂▂▂▃▃▃▃▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▇▇▇▇▇▇▇▇█████
holder	▁▁██████▁███████▁███████████████████████
trainer/global_step	▁▁▁▁▁▂▂▂▃▃▃▃▃▃▃▃▃▄▄▄▅▅▅▅▅▅▆▆▆▆▇▇▇▇▇▇████

Linear Val Loss	0.03043
Train Loss	0.0176
Val Loss	0.03043
epoch	643
holder	1
trainer/global_step	164863

Linear Val Loss	█▇▅▆▂▂▄▂█▂▃▁▄▃▄▁▁▂▂▄▂▂▂▂▅▂▃▂▂▃▂▁▂▃▁▂▂▁▂▂
Train Loss	▇▅▆▄▄█▃▂▄▃▃▃▂▂▂▄▂▃▂▃▃▂▂▂▄▂▂▁▂▂▂▂▂▂▂▁▃▃▂▂
Val Loss	█▅▂▂▂▂▂▄▂▂▂▃▂▂▁▂▃▂▁▂▂▂▂▁▂▁▁▁▁▁▂▁▁▂▁▂▁▁▁▁
epoch	▁▁▁▁▂▂▂▃▃▃▃▃▄▄▄▄▅▅▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇████
holder	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
trainer/global_step	▁▁▁▁▂▂▂▃▃▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▅▆▆▆▆▆▇▇▇█████

Linear Val Loss	0.0331
Train Loss	0.01642
Val Loss	0.0331
epoch	524
holder	1
trainer/global_step	134399

Linear Val Loss	▆▅█▂▂▂▂▂▂▂▂▂▂▂▂▂▂▁▂▂▂▂▂▁▁▁▂▁▁▂▂▁▂▂▁▁▂▁▂▁
Train Loss	▄▆█▂▃▂▃▃▃▃▂▂▂▂▂▃▂▂▂▂▁▂▃▃▂▁▂▂▁▁▂▂▁▁▂▂▂▁▂▁
Val Loss	█▂▂▂▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
epoch	▁▁▁▂▂▂▂▂▂▂▂▂▂▃▃▃▃▃▃▃▄▄▄▄▅▅▅▆▆▆▆▇▇▇▇▇▇▇██
holder	█▁█████████▁████████████████████████████
trainer/global_step	▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▃▃▃▃▃▄▄▅▅▅▅▅▅▆▆▆▆▇▇▇▇▇███

Linear Val Loss	0.03102
Train Loss	0.01326
Val Loss	0.03102
epoch	494
holder	1
trainer/global_step	126719

Linear Val Loss	█▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
Train Loss	█▄▄▄▃▅▃▃▃▂▃▂▂▂▁▂▁▂▁▂▁▂▂▂▁▂▂▂▂▂▁▁▂▃▁▂▂▂▁▂
Val Loss	█▃▂▂▂▁▂▂▂▁▂▁▁▁▁▁▂▁▁▂▁▁▁▁▁▂▁▂▃▂▂▁▂▂▂▁▁▁▂▂
epoch	▁▁▁▁▁▂▂▂▂▂▃▃▃▃▃▄▄▄▅▅▅▅▅▆▆▆▇▇▇▇▇▇▇▇▇▇▇███
holder	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
trainer/global_step	▁▁▂▂▂▂▂▂▃▃▃▄▄▄▄▄▄▅▅▅▆▆▆▆▆▆▆▇▇▇▇▇▇▇▇█████

Linear Val Loss	0.02886
Train Loss	0.01503
Val Loss	0.02886
epoch	494
holder	0
trainer/global_step	126719

Linear Val Loss	██▅▄▅▄▂▃▂▂▂▃▂▃▂▂▂▂▃▂▂▂▂▂▂▂▂▂▁▂▂▂▁▂▂▂▂▃▁▁
Train Loss	█▂▃▂▁▂▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
Val Loss	█▄▃▂▄▂▂▂▂▂▃▂▂▂▁▂▂▂▂▂▂▂▂▂▂▁▂▂▂▁▁▂▂▁▁▂▁▂▂▂
epoch	▁▁▁▁▁▂▂▂▂▂▃▃▃▃▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇▇█████
holder	█▁▁▁▁▁▁▁▁▁▁█▁▁▁█▁▁▁▁▁▁▁█▁▁▁▁▁▁▁▁▁█▁▁█▁▁▁
trainer/global_step	▁▁▁▁▁▁▂▂▂▂▂▂▃▃▃▃▃▃▃▃▄▄▄▄▄▄▅▅▆▆▆▆▇▇▇▇████

Linear Val Loss	0.02919
Train Loss	0.0128
Val Loss	0.02919
epoch	702
holder	0
trainer/global_step	179967

Linear Val Loss	█▅▅▃▂▃▁▂▁▃▂▂▁▂▂▂▂▁▁▂▂▁▁▂▁▁▁▂▁▁▁▁▁▁▁▁▁▁▁▁
Train Loss	█▇▅▃▆█▃▇▄▃▄▄▂▄▄▂▃▄▃▄▄▆▂▄▂▂▃▂▂▂▂▃▄▂▂▂▁▃▂▂
Val Loss	█▆▆▂▃▂▄▂▂▁▁▁▂▁▁▂▁▁▁▂▁▂▂▂▁▂▂▁▂▁▁▁▁▁▁▂▁▁▁▁
epoch	▁▁▁▁▁▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇▇▇▇██
holder	▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
trainer/global_step	▁▁▁▁▁▂▂▃▃▃▃▃▃▃▃▃▃▄▄▄▅▅▅▅▅▆▆▆▆▆▆▆▆▆▆▇▇███

Linear Val Loss	0.02505
Train Loss	0.00997
Val Loss	0.02505
epoch	708
holder	0
trainer/global_step	181503

Time to make figures documenting the system and the Fisher information measurements¶

Normalizing the Fisher information¶

The covariance component¶

Overall statistics of the Fisher information¶

Using FFT-like approach¶

Comparing the covariance component of the Fisher information¶

In summary,¶

Comparison to MSE¶

Does using other non-linearities produce smoother Fisher information?¶

Run history:

Run summary:

CELU¶

CELU: more neurons, no gradient clipping¶

Run history:

Run summary:

Big model, with gradient clipping¶

Run history:

Run summary:

Training a full set¶

What about models in eval mode?¶

Fisher informations¶

With covariance term¶

Overall:¶

Magnitude of the embedding¶

Same thing with ReLU¶

Explaining the plots¶

Same plots, single image¶

High Fisher¶

Is it easier to get an example of the behavior that we want using a linear embedding?¶

Model architecture search¶

Without dropout¶

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Run history:

Run summary:

Hindsight note:¶

Important note in hindsight:¶

Run history:

Run summary:

What about a powerful model, but with less training data?¶

Run history:

Run summary:

Putting Tanh on the outputs¶

Run history:

Run summary:

Same thing but with pytorch hardtanh¶

Run history:

Run summary:

Same thing but with higher bounds on the tanh¶

Another one, this time using a tanh with range that is slightly larger than out encoding requires.¶

Run history:

Run summary:

Some conclusions¶

Train vs test data¶

Fisher information¶

Interesting ideas here.¶

Hindsight note:¶

Epilogue: Training set Fisher information for the previous circular model.¶